package com.fm.magic.net.proxy;

import com.fm.base.log.Loggers;
import com.fm.base.net.proxy.HttpProxy;
import com.fm.base.util.Pair;
import com.fm.commons.net.http.HttpClient;
import com.fm.commons.net.http.HttpRequest;
import com.fm.commons.net.http.HttpRequestHandler;
import com.fm.commons.net.http.HttpResponse;
import com.fm.commons.net.http.converter.JsoupConverter;
import com.fm.commons.net.http.entity.FormDataProvider;
import com.fm.commons.util.StringUtil;
import org.apache.commons.codec.binary.Base64;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.net.URI;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ProxyCollector {
    public enum WebSite {
        YouDaiLi, KuaiDaiLi, XiCiDaiLi, XSDaiLi, CoolProxy, Spys
    }

    static class ProxyInfo {
        public String host;
        public int port;
        public String protocol;
        public HttpProxy.Type type;
        public String location;
        public WebSite website;

        @Override
        public boolean equals(Object obj) {
            return obj instanceof ProxyInfo
                && Objects.equals(host, ((ProxyInfo) obj).host)
                && port == ((ProxyInfo) obj).port;
        }

        @Override
        public int hashCode() {
            return (host + ":" + port).hashCode();
        }
    }

    public static Set<ProxyInfo> collect(WebSite webSite) {
        Set<ProxyInfo> cur;
        switch (webSite) {
            case YouDaiLi:
                cur = collectYouDaiLi();
                break;
            case KuaiDaiLi:
                cur = collectKuaiDaiLi();
                break;
            case XiCiDaiLi:
                cur = collectXiCiDaiLi();
                break;
            case XSDaiLi:
                cur = collectXSDaiLi();
                break;
            case CoolProxy:
                cur = collectCoolProxy();
                break;
            case Spys:
                cur = collectSpys();
                break;
            default:
                throw new RuntimeException();
        }
        if (cur != null) {
            Loggers.CONSOLE.info(webSite + " 采集到 " + cur.size() + " 个代理");
        } else {
            Loggers.CONSOLE.error(webSite + "采集失败");
        }
        return cur;
    }

    private static Set<ProxyInfo> collectYouDaiLi() {
        Set<ProxyInfo> all = new HashSet<>();
        try (HttpClient browser = new HttpClient()) {
            for (String module : new String[]{
                "http://www.youdaili.net/Daili/http",
                "http://www.youdaili.net/Daili/guonei",
                "http://www.youdaili.net/Daili/guowai"
            }) {
                HttpResponse response = browser.send(HttpRequest.get(URI.create(module)).build());
                if (response.getStatusCode() != 200) {
                    Loggers.CONSOLE.error("有代理采集失败 返回码:" + response.getStatusCode());
                    return null;
                } else {
                    Loggers.CONSOLE.info("start " + module);
                }

                Pattern p = Pattern.compile("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)@(\\S*?)#(【匿】)?(\\S*)");
                Elements as = response.convert(JsoupConverter.Instance).select("div.chunlist > ul > li:not(.line) > p > a");
                for (int i = 0; i < (as.size() > 3 ? 3 : as.size()); i++) {
                    String url = as.get(i).absUrl("href");
                    do {
                        try {
                            Thread.sleep(5000);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                        Loggers.CONSOLE.info("fetch " + url);
                        Document document = browser.process(HttpRequest.get(URI.create(url)).build(), new HttpRequestHandler<Document>() {
                            @Override
                            public Pair<Boolean, Document> handle(HttpResponse response, HttpClient client) throws Exception {
                                if (response.getStatusCode() == 200) {
                                    return new Pair<>(true, response.convert(JsoupConverter.Instance));
                                }
                                return new Pair<>(false, null);
                            }

                            @Override
                            public void doFail(Document model, HttpResponse response) {

                            }

                            @Override
                            public void doSuccess(Document model, HttpResponse response) {

                            }
                        }, 3);
                        if (document == null) {
                            System.out.println(new String(response.getContent()));
                            Loggers.CONSOLE.error("丢失网页:" + url);
                            break;
                        }
                        Elements content = document.select("div.content > p");
                        content.forEach(node -> {
                            Matcher m = p.matcher(node.text());
                            if (m.find()) {
                                ProxyInfo proxyInfo = new ProxyInfo();
                                proxyInfo.host = m.group(1);
                                proxyInfo.port = Integer.parseInt(m.group(2));
                                proxyInfo.protocol = m.group(3);
                                proxyInfo.type = m.group(4) == null ? HttpProxy.Type.Transparent : HttpProxy.Type.Anonymous;
                                proxyInfo.location = m.group(5);
                                proxyInfo.website = WebSite.YouDaiLi;
                                all.add(proxyInfo);
                            }
                        });
                        Element next = document.select("a:containsOwn(下一页):not([href=#])").first();
                        url = next == null ? null : next.absUrl("href");
                    } while (url != null);
                }
            }
        }
        return all;
    }

    private static Set<ProxyInfo> collectKuaiDaiLi() {
        Set<ProxyInfo> all = new HashSet<>();
        try (HttpClient browser = new HttpClient()) {
            for (String url : new String[]{
                "http://www.kuaidaili.com/free/inha/%d/",
                "http://www.kuaidaili.com/free/intr/%d/",
                "http://www.kuaidaili.com/free/outha/%d/",
                "http://www.kuaidaili.com/free/outtr/%d/"
            }) {
                int page = 0;
                while (page++ < 8) {
                    String pageUrl = String.format(url, page);
                    Loggers.CONSOLE.info("fetch " + pageUrl);
                    HttpResponse response = browser.send(HttpRequest.get(URI.create(pageUrl)).build());
                    if (response.getStatusCode() == 521) {
                        Document d = response.convert(JsoupConverter.Instance);
                        String js = d.select("script").first().data();
                        int index = js.indexOf("function");
                        String js_call = StringUtil.stringBetween(js.substring(0, index), "\"", "\"");
                        String function = js.substring(index).replace("eval(\"qo=eval;qo(po);\");", "return po;");
                        ScriptEngineManager sem = new ScriptEngineManager();
                        ScriptEngine se = sem.getEngineByName("javascript");
                        try {
                            se.eval(function);
                            String js_result = se.eval(js_call).toString();
                            response = browser.send(HttpRequest.get(URI.create("http://www.kuaidaili.com" + StringUtil.stringBetween(js_result, "\"", "\""))).build());
                        } catch (ScriptException e) {
                            Loggers.CONSOLE.error("", e);
                            return all;
                        }
                    }
                    if (response.getStatusCode() != 200) {
                        System.err.println(pageUrl + " 采集失败 返回码" + response.getStatusCode());
                        continue;
                    }
                    Document doc = response.convert(JsoupConverter.Instance);
                    Elements trs = doc.select("table.table > tbody > tr");
                    for (Element tr : trs) {
                        ProxyInfo proxyInfo = new ProxyInfo();
                        proxyInfo.host = tr.select("td:eq(0)").first().text();
                        proxyInfo.port = Integer.parseInt(tr.select("td:eq(1)").first().text());
                        proxyInfo.protocol = tr.select("td:eq(3)").first().text();
                        proxyInfo.type = "高匿名".equals(tr.select("td:eq(2)").first().text()) ? HttpProxy.Type.Anonymous : HttpProxy.Type.Transparent;
                        proxyInfo.location = tr.select("td:eq(4)").first().text();
                        proxyInfo.website = WebSite.KuaiDaiLi;
                        all.add(proxyInfo);
                    }
                    Thread.sleep(3000);
                }
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return all;
    }

    private static Set<ProxyInfo> collectXiCiDaiLi() {
        Set<ProxyInfo> all = new HashSet<>();
        try (HttpClient browser = new HttpClient(10)) {
            for (String url : new String[]{
                "http://www.xicidaili.com/nn/",
                "http://www.xicidaili.com/wn/",
                "http://www.xicidaili.com/nt/",
                "http://www.xicidaili.com/wt/"
            }) {
                int page = 0;
                while (page++ < 8) {
                    String pageUrl = url + page;
                    Loggers.CONSOLE.info("fetch " + pageUrl);
                    HttpResponse response = browser.send(HttpRequest.get(URI.create(pageUrl)).build());
                    if (response.getStatusCode() == 200) {
                        Document doc = response.convert(JsoupConverter.Instance);
                        Elements trs = doc.select("table#ip_list>tbody>tr:gt(0)");
                        for (Element tr : trs) {
                            ProxyInfo proxyInfo = new ProxyInfo();
                            proxyInfo.host = tr.select("td:eq(1)").first().text();
                            proxyInfo.port = Integer.parseInt(tr.select("td:eq(2)").first().text());
                            proxyInfo.protocol = tr.select("td:eq(5)").first().text();
                            proxyInfo.type = "高匿".equals(tr.select("td:eq(4)").first().text()) ? HttpProxy.Type.Anonymous : HttpProxy.Type.Transparent;
                            proxyInfo.location = tr.select("td:eq(3)").first().text();
                            proxyInfo.website = WebSite.XiCiDaiLi;
                            all.add(proxyInfo);
                        }
                    } else {
                        System.err.println(pageUrl + " 采集失败 返回码" + response.getStatusCode());
                    }
                    Thread.sleep(3000);
                }
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return all;
    }

    private static Set<ProxyInfo> collectXSDaiLi() {
        Set<ProxyInfo> all = new HashSet<>();
        try (HttpClient browser = new HttpClient()) {
            HttpRequest request = HttpRequest.get("https://www.xsdaili.com/").build();
            HttpResponse response = browser.send(request);
            if (response.getStatusCode() == 200) {
                Pattern p = Pattern.compile("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)@(\\S*?)#\\[(高匿名|透明)](\\S*)");
                Document dom = response.convert(JsoupConverter.Instance);
                Elements divs = dom.select("div.ips");
                for (int i = 0; i < 6; i++) {
                    Element a = divs.get(i).select("div.title>a").first();
                    String url = a.absUrl("href");
                    Loggers.CONSOLE.info("fetch " + url);
                    request = HttpRequest.get(url).build();
                    response = browser.send(request);
                    Thread.sleep(3000);
                    if (response.getStatusCode() == 200) {
                        Document doc = response.convert(JsoupConverter.Instance);
                        Element div = doc.select("div.cont").first();
                        div.childNodes().forEach(node -> {
                            if (node instanceof TextNode) {
                                Matcher m = p.matcher(node.toString());
                                if (m.find()) {
                                    ProxyInfo proxyInfo = new ProxyInfo();
                                    proxyInfo.host = m.group(1);
                                    proxyInfo.port = Integer.parseInt(m.group(2));
                                    proxyInfo.protocol = m.group(3);
                                    proxyInfo.type = "高匿名".equals(m.group(4)) ? HttpProxy.Type.Anonymous : HttpProxy.Type.Transparent;
                                    proxyInfo.location = m.group(5);
                                    proxyInfo.website = WebSite.XSDaiLi;
                                    all.add(proxyInfo);
                                }
                            }
                        });
                    } else {
                        System.err.println(url + " 采集失败 返回码" + response.getStatusCode());
                    }
                }
            } else {
                System.err.println("小舒代理 采集失败 返回码" + response.getStatusCode());
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return all;
    }

    private static Set<ProxyInfo> collectCoolProxy() {
        Set<ProxyInfo> all = new HashSet<>();
        try (HttpClient browser = new HttpClient()) {
            HttpResponse response = browser.send(HttpRequest.get(URI.create("http://www.cool-proxy.net/proxies/http_proxy_list/")).build());
            if (response.getStatusCode() == 200) {
                Document document = response.convert(JsoupConverter.Instance);
                int page = Integer.valueOf(document.select("th.pagination > span:not(.next)").last().text());
                for (int p = 1; p <= page; p++) {
                    Loggers.CONSOLE.info("fetch http://www.cool-proxy.net/proxies/http_proxy_list/page:" + p);
                    response = browser.send(HttpRequest.get(URI.create("http://www.cool-proxy.net/proxies/http_proxy_list/page:" + p)).build());
                    Thread.sleep(3000);
                    if (response.getStatusCode() == 200) {
                        Document doc = response.convert(JsoupConverter.Instance);
                        for (Element tr : doc.select("div#main > form[action=/proxies/http_proxy_list] ~ table tr")) {
                            Matcher matcher = Pattern.compile(".*str_rot13\\(\"(.*)\"\\).*").matcher(tr.select("script").html());
                            if (matcher.find()) {
                                String encodedHost = matcher.group(1);
                                StringBuilder decodedHost = new StringBuilder();
                                for (int ci = 0; ci < encodedHost.length(); ci++) {
                                    if (encodedHost.substring(ci, ci + 1).matches("[a-zA-Z]")) {
                                        char c = (char) (encodedHost.charAt(ci) + (encodedHost.substring(ci, ci + 1).toLowerCase().charAt(0) < 'n' ? 13 : -13));
                                        decodedHost.append(c);
                                    } else {
                                        decodedHost.append(encodedHost.charAt(ci));
                                    }
                                }
                                ProxyInfo proxy = new ProxyInfo();
                                proxy.host = new String(Base64.decodeBase64(decodedHost.toString())).replaceAll("\\s", "");
                                proxy.port = Integer.parseInt(tr.select("td:eq(1)").text());
                                proxy.protocol = "HTTP";
                                proxy.location = tr.select("td:eq(3)").text();
                                proxy.type = "Anonymous".equals(tr.select("td:eq(5)").text()) ? HttpProxy.Type.Anonymous : HttpProxy.Type.Transparent;
                                proxy.website = WebSite.CoolProxy;
                                all.add(proxy);
                            }
                        }
                    } else {
                        Loggers.CONSOLE.error("http://www.cool-proxy.net/proxies/http_proxy_list/page:" + p + " 访问失败");
                    }
                }
            } else {
                Loggers.CONSOLE.error("http://www.cool-proxy 访问失败");
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return all;
    }

    private static Set<ProxyInfo> collectSpys() {
        class Spys_js {
            private Map<String, String> map = new LinkedHashMap<>();

            private String y(int c) {
                int r = 60;
                String prefix = c < r ? "" : y(c / r);
                return prefix + ((c = c % r) > 35 ? (char) (c + 29) : (Long.toString(c, 36)));
            }

            private Spys_js(final String script1, final String script2) {
                String[] x = script2.split("\\^");
                int o = 60;
                Map<String, String> s = new TreeMap<>();
                while (o-- != 0) {
                    s.put(y(o), StringUtil.isEmpty(x[o]) ? y(o) : x[o]);
                }
                Pattern pattern = Pattern.compile("\\b\\w+\\b");
                Matcher matcher = pattern.matcher(script1);
                int index = 0;
                StringBuilder buf = new StringBuilder();
                while (matcher.find()) {
                    buf.append(script1.substring(index, matcher.start()));
                    buf.append(s.get(matcher.group(0)));
                    index = matcher.end();
                }
                buf.append(script1.substring(index));
                String result = buf.toString();
                for (String str : result.split(";")) {
                    if (!StringUtil.isEmpty(str)) {
                        String[] pair = str.split("=");
                        map.put(pair[0], pair[1]);
                    }
                }

                map.forEach((name, value) -> {
                    String[] split = value.split("\\^");
                    if (split.length > 1) {
                        Integer num = 0;
                        for (String symbol : split) {
                            String real = map.get(symbol);
                            num ^= Integer.parseInt(real == null ? symbol : real);
                        }
                        map.put(name, num.toString());
                    }
                });
            }

            private String get(String key) {
                return map.get(key);
            }
        }

        Set<ProxyInfo> all = new HashSet<>();
        try (HttpClient browser = new HttpClient()) {
            Map<String, String> data = new HashMap<>();
            data.put("xpp", "3");
            data.put("xf1", "0");
            data.put("xf2", "0");
            data.put("xf4", "0");
            for (String url : new String[]{
                "http://spys.ru/free-proxy-list/US/",
                "http://spys.ru/free-proxy-list/CN/",
                "http://spys.ru/free-proxy-list/BR/",
                "http://spys.ru/free-proxy-list/RU/",
                "http://spys.ru/free-proxy-list/TW/"}) {
                Loggers.CONSOLE.info("fetch " + url);
                HttpResponse response = browser.send(HttpRequest.post(URI.create(url), new FormDataProvider(data))
                    .setProxy("192.168.0.53", 1080)
                    .build());
                Thread.sleep(3000);
                if (response.getStatusCode() == 200) {
                    Document dom = response.convert(JsoupConverter.Instance);
                    String script1 = StringUtil.stringBetween(dom.select("body>script").first().data(), "x[o])}}return p}('", "',60,60,'");
                    String script2 = StringUtil.stringBetween(dom.select("body>script").first().data(), "',60,60,'", "'.split('\\u005e'),0,{}))");
                    Spys_js tool = new Spys_js(script1, script2);
                    Elements trs = dom.select("table>tbody>tr>td>table>tbody>tr[onmouseover]:not(:contains(SOCKS5))");
                    for (Element tr : trs) {
                        ProxyInfo proxyInfo = new ProxyInfo();
                        proxyInfo.host = tr.select("td:eq(0)>font.spy14").first().text();
                        String port_raw = StringUtil.stringBetween(
                            tr.select("td:eq(0)>font.spy14>script").first().toString(),
                            "\"<font class=spy2>:<\\/font>\"+",
                            ")</script>");
                        Pattern p = Pattern.compile("\\((\\S+?)\\)");
                        Matcher m = p.matcher(port_raw);
                        int port = 0;
                        while (m.find()) {
                            String info = m.group(1);
                            int port_num = 0;
                            for (String split : info.split("\\^")) {
                                port_num ^= Integer.parseInt(tool.get(split));
                            }
                            port = port * 10 + port_num;
                        }
                        proxyInfo.port = port;
                        proxyInfo.protocol = tr.select("td:eq(1)>a>font").first().text();
                        String type = tr.select("td:eq(2)>font").first().text();
                        proxyInfo.type = ("HIA".equals(type) || "ANM".equals(type)) ? HttpProxy.Type.Anonymous : HttpProxy.Type.Transparent;
                        proxyInfo.location = tr.select("td:eq(3)>font").first().ownText();
                        proxyInfo.website = WebSite.Spys;
                        all.add(proxyInfo);
                    }
                } else {
                    System.err.println("丢失网页:" + url);
                }
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return all;
    }
}
